{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from optimus import Optimus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Open Bumblebee: <a target='_blank' href ='https://app.hi-bumblebee.com/?session=ee8e887f-fac1-4555-aa54-90037f197099&key=hpmLDC4KO7hrtceEfn0KJMKaey1ZZlflNIYhS-7YU54=&view=0'>https://app.hi-bumblebee.com</a><div>If you really care about privacy get your keys in bumblebee.ini and put them<a href ='https://app.hi-bumblebee.com'> here</a></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\distributed\\dashboard\\core.py:79: UserWarning: \n",
      "Port 8787 is already in use. \n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the diagnostics dashboard on a random port instead.\n",
      "  warnings.warn(\"\\n\" + msg)\n"
     ]
    }
   ],
   "source": [
    "from optimus import Optimus\n",
    "op = Optimus(\"dask\", n_workers=1, threads_per_worker=8, processes=False, memory_limit=\"3G\", comm=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = op.load.file(\"data/crime.csv\").ext.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['INCIDENT_NUMBER',\n",
       " 'OFFENSE_CODE',\n",
       " 'OFFENSE_CODE_GROUP',\n",
       " 'OFFENSE_DESCRIPTION',\n",
       " 'DISTRICT',\n",
       " 'REPORTING_AREA',\n",
       " 'SHOOTING',\n",
       " 'OCCURRED_ON_DATE',\n",
       " 'YEAR',\n",
       " 'MONTH',\n",
       " 'DAY_OF_WEEK',\n",
       " 'HOUR',\n",
       " 'UCR_PART',\n",
       " 'STREET',\n",
       " 'Lat',\n",
       " 'Long',\n",
       " 'Location']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.cols.names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'file_name': 'crime.csv', 'mime_info': [{'mime': 'text/plain', 'encoding': 'iso-8859-1', 'file_ext': 'csv', 'file_type': 'csv', 'properties': {'delimiter': ',', 'doublequote': False, 'escapechar': None, 'lineterminator': '\\r\\n', 'quotechar': '\"', 'quoting': 0, 'skipinitialspace': False}}], 'profile': {'columns': {'INCIDENT_NUMBER': {'profiler_dtype': 'string'}, 'OFFENSE_CODE': {'profiler_dtype': 'zip_code'}, 'OFFENSE_CODE_GROUP': {'profiler_dtype': 'string'}, 'OFFENSE_DESCRIPTION': {'profiler_dtype': 'string'}, 'DISTRICT': {'profiler_dtype': 'string'}, 'REPORTING_AREA': {'profiler_dtype': 'int'}, 'SHOOTING': {'profiler_dtype': 'object'}, 'OCCURRED_ON_DATE': {'profiler_dtype': 'date'}, 'YEAR': {'profiler_dtype': 'int'}, 'MONTH': {'profiler_dtype': 'int'}, 'DAY_OF_WEEK': {'profiler_dtype': 'date'}, 'HOUR': {'profiler_dtype': 'int'}, 'UCR_PART': {'profiler_dtype': 'string'}, 'STREET': {'profiler_dtype': 'string'}, 'Lat': {'profiler_dtype': 'decimal'}, 'Long': {'profiler_dtype': 'decimal'}, 'Location': {'profiler_dtype': 'string'}}}, 'transformations': {'actions': [{'profiler_dtype': 'INCIDENT_NUMBER'}, {'profiler_dtype': 'OFFENSE_CODE'}, {'profiler_dtype': 'OFFENSE_CODE_GROUP'}, {'profiler_dtype': 'OFFENSE_DESCRIPTION'}, {'profiler_dtype': 'DISTRICT'}, {'profiler_dtype': 'REPORTING_AREA'}, {'profiler_dtype': 'SHOOTING'}, {'profiler_dtype': 'OCCURRED_ON_DATE'}, {'profiler_dtype': 'YEAR'}, {'profiler_dtype': 'MONTH'}, {'profiler_dtype': 'DAY_OF_WEEK'}, {'profiler_dtype': 'HOUR'}, {'profiler_dtype': 'UCR_PART'}, {'profiler_dtype': 'STREET'}, {'profiler_dtype': 'Lat'}, {'profiler_dtype': 'Long'}, {'profiler_dtype': 'Location'}]}}\n"
     ]
    }
   ],
   "source": [
    "cols_and_inferred_dtype = df.cols.infer_profiler_dtypes(\"*\")\n",
    "compute = True\n",
    "\n",
    "mismatch = df.cols.count_mismatch(cols_and_inferred_dtype, infer=True, compute=compute)\n",
    "df.cols.set_profiler_dtypes(cols_and_inferred_dtype)\n",
    "print(df.meta.get())\n",
    "               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'columns': {'INCIDENT_NUMBER': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'frequency': [{'value': 'I162030584', 'count': 13},\n",
       "     {'value': 'I152080623', 'count': 11},\n",
       "     {'value': 'I172013170', 'count': 10},\n",
       "     {'value': 'I182065208', 'count': 10},\n",
       "     {'value': 'I172096394', 'count': 10},\n",
       "     {'value': 'I162071327', 'count': 9},\n",
       "     {'value': 'I162001871', 'count': 9},\n",
       "     {'value': 'I172056883', 'count': 9},\n",
       "     {'value': 'I172022524', 'count': 9},\n",
       "     {'value': 'I172054429', 'count': 9},\n",
       "     {'value': 'I162098170', 'count': 9},\n",
       "     {'value': 'I162078338', 'count': 8},\n",
       "     {'value': 'I162074826', 'count': 8},\n",
       "     {'value': 'I162090278', 'count': 8},\n",
       "     {'value': 'I172069723', 'count': 8},\n",
       "     {'value': 'I162022140', 'count': 8},\n",
       "     {'value': 'I130041200-00', 'count': 8},\n",
       "     {'value': 'I162087224', 'count': 8},\n",
       "     {'value': 'I152076465', 'count': 8},\n",
       "     {'value': 'I162056703', 'count': 8},\n",
       "     {'value': 'I162064331', 'count': 8},\n",
       "     {'value': 'I162082917', 'count': 8},\n",
       "     {'value': 'I152105431', 'count': 8},\n",
       "     {'value': 'I172053616', 'count': 8},\n",
       "     {'value': 'I152101399', 'count': 7},\n",
       "     {'value': 'I152095733', 'count': 7},\n",
       "     {'value': 'I162054378', 'count': 7},\n",
       "     {'value': 'I152067057', 'count': 7},\n",
       "     {'value': 'I152096998', 'count': 7},\n",
       "     {'value': 'I172018004', 'count': 7},\n",
       "     {'value': 'I162083089', 'count': 7},\n",
       "     {'value': 'I162045680', 'count': 7},\n",
       "     {'value': 'I152071480', 'count': 7}],\n",
       "    'count_uniques': 282517},\n",
       "   'dtype': 'object'},\n",
       "  'OFFENSE_CODE': {'profiler_dtype': 'zip_code',\n",
       "   'stats': {'mismatch': 251,\n",
       "    'missing': 0,\n",
       "    'match': 318822,\n",
       "    'frequency': [{'value': '03006', 'count': 18783},\n",
       "     {'value': '03115', 'count': 18746},\n",
       "     {'value': '03831', 'count': 16323},\n",
       "     {'value': '01402', 'count': 15152},\n",
       "     {'value': '00802', 'count': 14791},\n",
       "     {'value': '03301', 'count': 13099},\n",
       "     {'value': '03410', 'count': 11287},\n",
       "     {'value': '03114', 'count': 11123},\n",
       "     {'value': '00617', 'count': 9069},\n",
       "     {'value': '02647', 'count': 9039},\n",
       "     {'value': '00614', 'count': 8893},\n",
       "     {'value': '03201', 'count': 8892},\n",
       "     {'value': '03125', 'count': 8389},\n",
       "     {'value': '00613', 'count': 7949},\n",
       "     {'value': '03802', 'count': 6557},\n",
       "     {'value': '00619', 'count': 5963},\n",
       "     {'value': '03803', 'count': 5131},\n",
       "     {'value': '00413', 'count': 4886},\n",
       "     {'value': '01102', 'count': 4413},\n",
       "     {'value': '03502', 'count': 4365},\n",
       "     {'value': '02629', 'count': 4007},\n",
       "     {'value': '03501', 'count': 3766},\n",
       "     {'value': '03207', 'count': 3698},\n",
       "     {'value': '00724', 'count': 3629},\n",
       "     {'value': '02610', 'count': 3245},\n",
       "     {'value': '01106', 'count': 3147},\n",
       "     {'value': '00301', 'count': 3056},\n",
       "     {'value': '03801', 'count': 2925},\n",
       "     {'value': '00423', 'count': 2910},\n",
       "     {'value': '02900', 'count': 2894},\n",
       "     {'value': '02907', 'count': 2616},\n",
       "     {'value': '00520', 'count': 2585},\n",
       "     {'value': '01849', 'count': 2584}],\n",
       "    'count_uniques': 263},\n",
       "   'dtype': 'object'},\n",
       "  'OFFENSE_CODE_GROUP': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'frequency': [{'value': 'Motor Vehicle Accident Response', 'count': 37132},\n",
       "     {'value': 'Larceny', 'count': 25935},\n",
       "     {'value': 'Medical Assistance', 'count': 23540},\n",
       "     {'value': 'Investigate Person', 'count': 18750},\n",
       "     {'value': 'Other', 'count': 18075},\n",
       "     {'value': 'Drug Violation', 'count': 16548},\n",
       "     {'value': 'Simple Assault', 'count': 15826},\n",
       "     {'value': 'Vandalism', 'count': 15415},\n",
       "     {'value': 'Verbal Disputes', 'count': 13099},\n",
       "     {'value': 'Towed', 'count': 11287},\n",
       "     {'value': 'Investigate Property', 'count': 11124},\n",
       "     {'value': 'Larceny From Motor Vehicle', 'count': 10847},\n",
       "     {'value': 'Property Lost', 'count': 9751},\n",
       "     {'value': 'Warrant Arrests', 'count': 8407},\n",
       "     {'value': 'Aggravated Assault', 'count': 7807},\n",
       "     {'value': 'Violations', 'count': 6095},\n",
       "     {'value': 'Fraud', 'count': 5829},\n",
       "     {'value': 'Residential Burglary', 'count': 5606},\n",
       "     {'value': 'Missing Person Located', 'count': 4958},\n",
       "     {'value': 'Auto Theft', 'count': 4851},\n",
       "     {'value': 'Robbery', 'count': 4624},\n",
       "     {'value': 'Harassment', 'count': 4007},\n",
       "     {'value': 'Property Found', 'count': 3925},\n",
       "     {'value': 'Missing Person Reported', 'count': 3797},\n",
       "     {'value': 'Confidence Games', 'count': 3147},\n",
       "     {'value': 'Police Service Incidents', 'count': 2781},\n",
       "     {'value': 'Disorderly Conduct', 'count': 2611},\n",
       "     {'value': 'Fire Related Reports', 'count': 1920},\n",
       "     {'value': 'Firearm Violations', 'count': 1777},\n",
       "     {'value': 'License Violation', 'count': 1701},\n",
       "     {'value': 'Restraining Order Violations', 'count': 1607},\n",
       "     {'value': 'Recovered Stolen Property', 'count': 1455},\n",
       "     {'value': 'Counterfeiting', 'count': 1454}],\n",
       "    'count_uniques': 67},\n",
       "   'dtype': 'object'},\n",
       "  'OFFENSE_DESCRIPTION': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'frequency': [{'value': 'SICK/INJURED/MEDICAL - PERSON', 'count': 18783},\n",
       "     {'value': 'INVESTIGATE PERSON', 'count': 18754},\n",
       "     {'value': 'M/V - LEAVING SCENE - PROPERTY DAMAGE', 'count': 16323},\n",
       "     {'value': 'VANDALISM', 'count': 15154},\n",
       "     {'value': 'ASSAULT SIMPLE - BATTERY', 'count': 14791},\n",
       "     {'value': 'VERBAL DISPUTE', 'count': 13099},\n",
       "     {'value': 'TOWED MOTOR VEHICLE', 'count': 11287},\n",
       "     {'value': 'INVESTIGATE PROPERTY', 'count': 11124},\n",
       "     {'value': 'LARCENY THEFT FROM BUILDING', 'count': 9069},\n",
       "     {'value': 'THREATS TO DO BODILY HARM', 'count': 9042},\n",
       "     {'value': 'LARCENY THEFT FROM MV - NON-ACCESSORY', 'count': 8893},\n",
       "     {'value': 'PROPERTY - LOST', 'count': 8893},\n",
       "     {'value': 'WARRANT ARREST', 'count': 8407},\n",
       "     {'value': 'LARCENY SHOPLIFTING', 'count': 7949},\n",
       "     {'value': 'M/V ACCIDENT - PROPERTY \\xa0DAMAGE', 'count': 6557},\n",
       "     {'value': 'LARCENY ALL OTHERS', 'count': 5963},\n",
       "     {'value': 'M/V ACCIDENT - PERSONAL INJURY', 'count': 5131},\n",
       "     {'value': 'ASSAULT - AGGRAVATED - BATTERY', 'count': 4886},\n",
       "     {'value': 'FRAUD - FALSE PRETENSE / SCHEME', 'count': 4413},\n",
       "     {'value': 'MISSING PERSON - LOCATED', 'count': 4365},\n",
       "     {'value': 'HARASSMENT', 'count': 4007},\n",
       "     {'value': 'MISSING PERSON', 'count': 3766},\n",
       "     {'value': 'PROPERTY - FOUND', 'count': 3698},\n",
       "     {'value': 'AUTO THEFT', 'count': 3630},\n",
       "     {'value': 'TRESPASSING', 'count': 3254},\n",
       "     {'value': 'FRAUD - CREDIT CARD / ATM FRAUD', 'count': 3147},\n",
       "     {'value': 'ROBBERY - STREET', 'count': 3056},\n",
       "     {'value': 'M/V ACCIDENT - OTHER', 'count': 2925},\n",
       "     {'value': 'ASSAULT - AGGRAVATED', 'count': 2910},\n",
       "     {'value': 'VAL - VIOLATION OF AUTO LAW - OTHER', 'count': 2894},\n",
       "     {'value': 'VAL - OPERATING AFTER REV/SUSP.', 'count': 2618},\n",
       "     {'value': 'DRUGS - POSS CLASS B - COCAINE, ETC.', 'count': 2591},\n",
       "     {'value': 'BURGLARY - RESIDENTIAL - FORCE', 'count': 2585}],\n",
       "    'count_uniques': 244},\n",
       "   'dtype': 'object'},\n",
       "  'DISTRICT': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 1765,\n",
       "    'match': 317308,\n",
       "    'frequency': [{'value': 'B2', 'count': 49945},\n",
       "     {'value': 'C11', 'count': 42530},\n",
       "     {'value': 'D4', 'count': 41915},\n",
       "     {'value': 'A1', 'count': 35717},\n",
       "     {'value': 'B3', 'count': 35442},\n",
       "     {'value': 'C6', 'count': 23460},\n",
       "     {'value': 'D14', 'count': 20127},\n",
       "     {'value': 'E13', 'count': 17536},\n",
       "     {'value': 'E18', 'count': 17348},\n",
       "     {'value': 'A7', 'count': 13544},\n",
       "     {'value': 'E5', 'count': 13239},\n",
       "     {'value': 'A15', 'count': 6505}],\n",
       "    'count_uniques': 12},\n",
       "   'dtype': 'object'},\n",
       "  'REPORTING_AREA': {'profiler_dtype': 'int',\n",
       "   'stats': {'mismatch': 20250, 'missing': 0, 'match': 298823},\n",
       "   'dtype': 'object'},\n",
       "  'SHOOTING': {'profiler_dtype': 'object',\n",
       "   'stats': {'mismatch': 1019,\n",
       "    'missing': 318054,\n",
       "    'match': 0,\n",
       "    'frequency': [{'value': 'Y', 'count': 1019}],\n",
       "    'count_uniques': 1},\n",
       "   'dtype': 'object'},\n",
       "  'OCCURRED_ON_DATE': {'profiler_dtype': 'date',\n",
       "   'stats': {'mismatch': 319073,\n",
       "    'missing': 0,\n",
       "    'match': 0,\n",
       "    'frequency': [{'value': '2017-06-01 00:00:00', 'count': 29},\n",
       "     {'value': '2015-07-01 00:00:00', 'count': 27},\n",
       "     {'value': '2016-08-01 00:00:00', 'count': 27},\n",
       "     {'value': '2015-06-18 05:00:00', 'count': 22},\n",
       "     {'value': '2017-08-01 00:00:00', 'count': 22},\n",
       "     {'value': '2017-01-01 00:00:00', 'count': 21},\n",
       "     {'value': '2016-04-01 00:00:00', 'count': 20},\n",
       "     {'value': '2017-05-01 00:00:00', 'count': 20},\n",
       "     {'value': '2015-12-07 11:38:00', 'count': 20},\n",
       "     {'value': '2017-04-01 00:00:00', 'count': 19},\n",
       "     {'value': '2016-09-01 00:00:00', 'count': 19},\n",
       "     {'value': '2018-01-01 00:00:00', 'count': 18},\n",
       "     {'value': '2016-11-01 00:00:00', 'count': 18},\n",
       "     {'value': '2018-06-04 12:40:00', 'count': 18},\n",
       "     {'value': '2015-12-01 00:00:00', 'count': 18},\n",
       "     {'value': '2017-07-05 00:00:00', 'count': 18},\n",
       "     {'value': '2017-11-01 00:00:00', 'count': 17},\n",
       "     {'value': '2018-02-01 00:00:00', 'count': 16},\n",
       "     {'value': '2016-02-01 00:00:00', 'count': 16},\n",
       "     {'value': '2018-07-19 00:00:00', 'count': 16},\n",
       "     {'value': '2017-03-01 00:00:00', 'count': 15},\n",
       "     {'value': '2018-03-07 06:00:00', 'count': 15},\n",
       "     {'value': '2018-08-18 08:45:00', 'count': 15},\n",
       "     {'value': '2017-10-01 00:00:00', 'count': 15},\n",
       "     {'value': '2016-12-15 06:00:00', 'count': 14},\n",
       "     {'value': '2015-06-20 00:00:00', 'count': 14},\n",
       "     {'value': '2016-06-24 17:30:00', 'count': 14},\n",
       "     {'value': '2016-11-10 16:00:00', 'count': 13},\n",
       "     {'value': '2018-04-02 00:00:00', 'count': 13},\n",
       "     {'value': '2015-12-22 20:25:00', 'count': 13},\n",
       "     {'value': '2018-01-03 18:00:00', 'count': 13},\n",
       "     {'value': '2015-07-17 00:00:00', 'count': 13},\n",
       "     {'value': '2016-04-20 11:07:00', 'count': 13}],\n",
       "    'count_uniques': 233229},\n",
       "   'dtype': 'object'},\n",
       "  'YEAR': {'profiler_dtype': 'int',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'hist': [{'lower': 2015.0, 'upper': 2015.09375, 'count': 53388},\n",
       "     {'lower': 2015.09375, 'upper': 2015.1875, 'count': 0},\n",
       "     {'lower': 2015.1875, 'upper': 2015.28125, 'count': 0},\n",
       "     {'lower': 2015.28125, 'upper': 2015.375, 'count': 0},\n",
       "     {'lower': 2015.375, 'upper': 2015.46875, 'count': 0},\n",
       "     {'lower': 2015.46875, 'upper': 2015.5625, 'count': 0},\n",
       "     {'lower': 2015.5625, 'upper': 2015.65625, 'count': 0},\n",
       "     {'lower': 2015.65625, 'upper': 2015.75, 'count': 0},\n",
       "     {'lower': 2015.75, 'upper': 2015.84375, 'count': 0},\n",
       "     {'lower': 2015.84375, 'upper': 2015.9375, 'count': 0},\n",
       "     {'lower': 2015.9375, 'upper': 2016.03125, 'count': 99114},\n",
       "     {'lower': 2016.03125, 'upper': 2016.125, 'count': 0},\n",
       "     {'lower': 2016.125, 'upper': 2016.21875, 'count': 0},\n",
       "     {'lower': 2016.21875, 'upper': 2016.3125, 'count': 0},\n",
       "     {'lower': 2016.3125, 'upper': 2016.40625, 'count': 0},\n",
       "     {'lower': 2016.40625, 'upper': 2016.5, 'count': 0},\n",
       "     {'lower': 2016.5, 'upper': 2016.59375, 'count': 0},\n",
       "     {'lower': 2016.59375, 'upper': 2016.6875, 'count': 0},\n",
       "     {'lower': 2016.6875, 'upper': 2016.78125, 'count': 0},\n",
       "     {'lower': 2016.78125, 'upper': 2016.875, 'count': 0},\n",
       "     {'lower': 2016.875, 'upper': 2016.96875, 'count': 0},\n",
       "     {'lower': 2016.96875, 'upper': 2017.0625, 'count': 100886},\n",
       "     {'lower': 2017.0625, 'upper': 2017.15625, 'count': 0},\n",
       "     {'lower': 2017.15625, 'upper': 2017.25, 'count': 0},\n",
       "     {'lower': 2017.25, 'upper': 2017.34375, 'count': 0},\n",
       "     {'lower': 2017.34375, 'upper': 2017.4375, 'count': 0},\n",
       "     {'lower': 2017.4375, 'upper': 2017.53125, 'count': 0},\n",
       "     {'lower': 2017.53125, 'upper': 2017.625, 'count': 0},\n",
       "     {'lower': 2017.625, 'upper': 2017.71875, 'count': 0},\n",
       "     {'lower': 2017.71875, 'upper': 2017.8125, 'count': 0},\n",
       "     {'lower': 2017.8125, 'upper': 2017.90625, 'count': 0},\n",
       "     {'lower': 2017.90625, 'upper': 2018.0, 'count': 65685}],\n",
       "    'count_uniques': 4},\n",
       "   'dtype': 'object'},\n",
       "  'MONTH': {'profiler_dtype': 'int',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'hist': [{'lower': 1.0, 'upper': 1.34375, 'count': 23610},\n",
       "     {'lower': 1.34375, 'upper': 1.6875, 'count': 0},\n",
       "     {'lower': 1.6875, 'upper': 2.03125, 'count': 21653},\n",
       "     {'lower': 2.03125, 'upper': 2.375, 'count': 0},\n",
       "     {'lower': 2.375, 'upper': 2.71875, 'count': 0},\n",
       "     {'lower': 2.71875, 'upper': 3.0625, 'count': 24146},\n",
       "     {'lower': 3.0625, 'upper': 3.40625, 'count': 0},\n",
       "     {'lower': 3.40625, 'upper': 3.75, 'count': 0},\n",
       "     {'lower': 3.75, 'upper': 4.09375, 'count': 24086},\n",
       "     {'lower': 4.09375, 'upper': 4.4375, 'count': 0},\n",
       "     {'lower': 4.4375, 'upper': 4.78125, 'count': 0},\n",
       "     {'lower': 4.78125, 'upper': 5.125, 'count': 26199},\n",
       "     {'lower': 5.125, 'upper': 5.46875, 'count': 0},\n",
       "     {'lower': 5.46875, 'upper': 5.8125, 'count': 0},\n",
       "     {'lower': 5.8125, 'upper': 6.15625, 'count': 30568},\n",
       "     {'lower': 6.15625, 'upper': 6.5, 'count': 0},\n",
       "     {'lower': 6.5, 'upper': 6.84375, 'count': 0},\n",
       "     {'lower': 6.84375, 'upper': 7.1875, 'count': 34556},\n",
       "     {'lower': 7.1875, 'upper': 7.53125, 'count': 0},\n",
       "     {'lower': 7.53125, 'upper': 7.875, 'count': 0},\n",
       "     {'lower': 7.875, 'upper': 8.21875, 'count': 34823},\n",
       "     {'lower': 8.21875, 'upper': 8.5625, 'count': 0},\n",
       "     {'lower': 8.5625, 'upper': 8.90625, 'count': 0},\n",
       "     {'lower': 8.90625, 'upper': 9.25, 'count': 26543},\n",
       "     {'lower': 9.25, 'upper': 9.59375, 'count': 0},\n",
       "     {'lower': 9.59375, 'upper': 9.9375, 'count': 0},\n",
       "     {'lower': 9.9375, 'upper': 10.28125, 'count': 25737},\n",
       "     {'lower': 10.28125, 'upper': 10.625, 'count': 0},\n",
       "     {'lower': 10.625, 'upper': 10.96875, 'count': 0},\n",
       "     {'lower': 10.96875, 'upper': 11.3125, 'count': 23675},\n",
       "     {'lower': 11.3125, 'upper': 11.65625, 'count': 0},\n",
       "     {'lower': 11.65625, 'upper': 12.0, 'count': 23477}],\n",
       "    'count_uniques': 12},\n",
       "   'dtype': 'object'},\n",
       "  'DAY_OF_WEEK': {'profiler_dtype': 'date',\n",
       "   'stats': {'mismatch': 319073,\n",
       "    'missing': 0,\n",
       "    'match': 0,\n",
       "    'frequency': [{'value': 'Friday', 'count': 48495},\n",
       "     {'value': 'Wednesday', 'count': 46729},\n",
       "     {'value': 'Thursday', 'count': 46656},\n",
       "     {'value': 'Tuesday', 'count': 46383},\n",
       "     {'value': 'Monday', 'count': 45679},\n",
       "     {'value': 'Saturday', 'count': 44818},\n",
       "     {'value': 'Sunday', 'count': 40313}],\n",
       "    'count_uniques': 7},\n",
       "   'dtype': 'object'},\n",
       "  'HOUR': {'profiler_dtype': 'int',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'hist': [{'lower': 0.0, 'upper': 0.71875, 'count': 15106},\n",
       "     {'lower': 0.71875, 'upper': 1.4375, 'count': 9269},\n",
       "     {'lower': 1.4375, 'upper': 2.15625, 'count': 7693},\n",
       "     {'lower': 2.15625, 'upper': 2.875, 'count': 0},\n",
       "     {'lower': 2.875, 'upper': 3.59375, 'count': 4589},\n",
       "     {'lower': 3.59375, 'upper': 4.3125, 'count': 3408},\n",
       "     {'lower': 4.3125, 'upper': 5.03125, 'count': 3311},\n",
       "     {'lower': 5.03125, 'upper': 5.75, 'count': 0},\n",
       "     {'lower': 5.75, 'upper': 6.46875, 'count': 5026},\n",
       "     {'lower': 6.46875, 'upper': 7.1875, 'count': 8850},\n",
       "     {'lower': 7.1875, 'upper': 7.90625, 'count': 0},\n",
       "     {'lower': 7.90625, 'upper': 8.625, 'count': 13043},\n",
       "     {'lower': 8.625, 'upper': 9.34375, 'count': 14740},\n",
       "     {'lower': 9.34375, 'upper': 10.0625, 'count': 16347},\n",
       "     {'lower': 10.0625, 'upper': 10.78125, 'count': 0},\n",
       "     {'lower': 10.78125, 'upper': 11.5, 'count': 16448},\n",
       "     {'lower': 11.5, 'upper': 12.21875, 'count': 18679},\n",
       "     {'lower': 12.21875, 'upper': 12.9375, 'count': 0},\n",
       "     {'lower': 12.9375, 'upper': 13.65625, 'count': 16846},\n",
       "     {'lower': 13.65625, 'upper': 14.375, 'count': 17189},\n",
       "     {'lower': 14.375, 'upper': 15.09375, 'count': 16522},\n",
       "     {'lower': 15.09375, 'upper': 15.8125, 'count': 0},\n",
       "     {'lower': 15.8125, 'upper': 16.53125, 'count': 19871},\n",
       "     {'lower': 16.53125, 'upper': 17.25, 'count': 20763},\n",
       "     {'lower': 17.25, 'upper': 17.96875, 'count': 0},\n",
       "     {'lower': 17.96875, 'upper': 18.6875, 'count': 20302},\n",
       "     {'lower': 18.6875, 'upper': 19.40625, 'count': 17588},\n",
       "     {'lower': 19.40625, 'upper': 20.125, 'count': 15850},\n",
       "     {'lower': 20.125, 'upper': 20.84375, 'count': 0},\n",
       "     {'lower': 20.84375, 'upper': 21.5625, 'count': 14111},\n",
       "     {'lower': 21.5625, 'upper': 22.28125, 'count': 12926},\n",
       "     {'lower': 22.28125, 'upper': 23.0, 'count': 10596}],\n",
       "    'count_uniques': 24},\n",
       "   'dtype': 'object'},\n",
       "  'UCR_PART': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 90,\n",
       "    'match': 318983,\n",
       "    'frequency': [{'value': 'Part Three', 'count': 158553},\n",
       "     {'value': 'Part Two', 'count': 97569},\n",
       "     {'value': 'Part One', 'count': 61629},\n",
       "     {'value': 'Other', 'count': 1232}],\n",
       "    'count_uniques': 4},\n",
       "   'dtype': 'object'},\n",
       "  'STREET': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 10871,\n",
       "    'match': 308202,\n",
       "    'frequency': [{'value': 'WASHINGTON ST', 'count': 14194},\n",
       "     {'value': 'BLUE HILL AVE', 'count': 7794},\n",
       "     {'value': 'BOYLSTON ST', 'count': 7221},\n",
       "     {'value': 'DORCHESTER AVE', 'count': 5149},\n",
       "     {'value': 'TREMONT ST', 'count': 4796},\n",
       "     {'value': 'MASSACHUSETTS AVE', 'count': 4708},\n",
       "     {'value': 'HARRISON AVE', 'count': 4609},\n",
       "     {'value': 'CENTRE ST', 'count': 4383},\n",
       "     {'value': 'COMMONWEALTH AVE', 'count': 4134},\n",
       "     {'value': 'HYDE PARK AVE', 'count': 3470},\n",
       "     {'value': 'COLUMBIA RD', 'count': 3195},\n",
       "     {'value': 'HUNTINGTON AVE', 'count': 2989},\n",
       "     {'value': 'RIVER ST', 'count': 2831},\n",
       "     {'value': 'DUDLEY ST', 'count': 2352},\n",
       "     {'value': 'WARREN ST', 'count': 2336},\n",
       "     {'value': 'COLUMBUS AVE', 'count': 2304},\n",
       "     {'value': 'ADAMS ST', 'count': 2042},\n",
       "     {'value': 'BEACON ST', 'count': 1966},\n",
       "     {'value': 'NEWBURY ST', 'count': 1889},\n",
       "     {'value': 'CAMBRIDGE ST', 'count': 1670},\n",
       "     {'value': 'BOWDOIN ST', 'count': 1656},\n",
       "     {'value': 'AMERICAN LEGION HWY', 'count': 1631},\n",
       "     {'value': 'W BROADWAY', 'count': 1630},\n",
       "     {'value': 'GENEVA AVE', 'count': 1628},\n",
       "     {'value': 'ALBANY ST', 'count': 1429},\n",
       "     {'value': 'ALLSTATE RD', 'count': 1296},\n",
       "     {'value': 'SUMMER ST', 'count': 1250},\n",
       "     {'value': 'HANCOCK ST', 'count': 1246},\n",
       "     {'value': 'BORDER ST', 'count': 1213},\n",
       "     {'value': 'NEW SUDBURY ST', 'count': 1213},\n",
       "     {'value': 'CUMMINS HWY', 'count': 1151},\n",
       "     {'value': 'NORFOLK ST', 'count': 1115},\n",
       "     {'value': 'BENNINGTON ST', 'count': 1084}],\n",
       "    'count_uniques': 4657},\n",
       "   'dtype': 'object'},\n",
       "  'Lat': {'profiler_dtype': 'decimal',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'hist': [{'lower': -1.0, 'upper': 0.3560950493749999, 'count': 745},\n",
       "     {'lower': 0.3560950493749999, 'upper': 1.7121900987499998, 'count': 0},\n",
       "     {'lower': 1.7121900987499998, 'upper': 3.0682851481249998, 'count': 0},\n",
       "     {'lower': 3.0682851481249998, 'upper': 4.4243801975, 'count': 0},\n",
       "     {'lower': 4.4243801975, 'upper': 5.780475246875, 'count': 0},\n",
       "     {'lower': 5.780475246875, 'upper': 7.1365702962499995, 'count': 0},\n",
       "     {'lower': 7.1365702962499995, 'upper': 8.492665345625, 'count': 0},\n",
       "     {'lower': 8.492665345625, 'upper': 9.848760395, 'count': 0},\n",
       "     {'lower': 9.848760395, 'upper': 11.204855444375, 'count': 0},\n",
       "     {'lower': 11.204855444375, 'upper': 12.56095049375, 'count': 0},\n",
       "     {'lower': 12.56095049375, 'upper': 13.917045543124999, 'count': 0},\n",
       "     {'lower': 13.917045543124999, 'upper': 15.273140592499999, 'count': 0},\n",
       "     {'lower': 15.273140592499999, 'upper': 16.629235641875, 'count': 0},\n",
       "     {'lower': 16.629235641875, 'upper': 17.98533069125, 'count': 0},\n",
       "     {'lower': 17.98533069125, 'upper': 19.341425740625, 'count': 0},\n",
       "     {'lower': 19.341425740625, 'upper': 20.69752079, 'count': 0},\n",
       "     {'lower': 20.69752079, 'upper': 22.053615839375, 'count': 0},\n",
       "     {'lower': 22.053615839375, 'upper': 23.40971088875, 'count': 0},\n",
       "     {'lower': 23.40971088875, 'upper': 24.765805938125, 'count': 0},\n",
       "     {'lower': 24.765805938125, 'upper': 26.1219009875, 'count': 0},\n",
       "     {'lower': 26.1219009875, 'upper': 27.477996036875, 'count': 0},\n",
       "     {'lower': 27.477996036875, 'upper': 28.834091086249998, 'count': 0},\n",
       "     {'lower': 28.834091086249998, 'upper': 30.190186135624998, 'count': 0},\n",
       "     {'lower': 30.190186135624998, 'upper': 31.546281184999998, 'count': 0},\n",
       "     {'lower': 31.546281184999998, 'upper': 32.902376234375, 'count': 0},\n",
       "     {'lower': 32.902376234375, 'upper': 34.25847128375, 'count': 0},\n",
       "     {'lower': 34.25847128375, 'upper': 35.614566333125, 'count': 0},\n",
       "     {'lower': 35.614566333125, 'upper': 36.9706613825, 'count': 0},\n",
       "     {'lower': 36.9706613825, 'upper': 38.326756431875, 'count': 0},\n",
       "     {'lower': 38.326756431875, 'upper': 39.68285148125, 'count': 0},\n",
       "     {'lower': 39.68285148125, 'upper': 41.038946530625, 'count': 0},\n",
       "     {'lower': 41.038946530625, 'upper': 42.39504158, 'count': 298329}],\n",
       "    'count_uniques': 18178},\n",
       "   'dtype': 'object'},\n",
       "  'Long': {'profiler_dtype': 'decimal',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'hist': [{'lower': -71.17867378,\n",
       "      'upper': -68.985590224375,\n",
       "      'count': 298329},\n",
       "     {'lower': -68.985590224375, 'upper': -66.79250666875, 'count': 0},\n",
       "     {'lower': -66.79250666875, 'upper': -64.599423113125, 'count': 0},\n",
       "     {'lower': -64.599423113125, 'upper': -62.4063395575, 'count': 0},\n",
       "     {'lower': -62.4063395575, 'upper': -60.213256001874996, 'count': 0},\n",
       "     {'lower': -60.213256001874996, 'upper': -58.02017244625, 'count': 0},\n",
       "     {'lower': -58.02017244625, 'upper': -55.827088890625, 'count': 0},\n",
       "     {'lower': -55.827088890625, 'upper': -53.634005335, 'count': 0},\n",
       "     {'lower': -53.634005335, 'upper': -51.440921779374996, 'count': 0},\n",
       "     {'lower': -51.440921779374996, 'upper': -49.247838223749994, 'count': 0},\n",
       "     {'lower': -49.247838223749994, 'upper': -47.054754668125, 'count': 0},\n",
       "     {'lower': -47.054754668125, 'upper': -44.8616711125, 'count': 0},\n",
       "     {'lower': -44.8616711125, 'upper': -42.668587556874996, 'count': 0},\n",
       "     {'lower': -42.668587556874996, 'upper': -40.47550400125, 'count': 0},\n",
       "     {'lower': -40.47550400125, 'upper': -38.282420445625, 'count': 0},\n",
       "     {'lower': -38.282420445625, 'upper': -36.08933689, 'count': 0},\n",
       "     {'lower': -36.08933689, 'upper': -33.896253334375, 'count': 0},\n",
       "     {'lower': -33.896253334375, 'upper': -31.703169778749995, 'count': 0},\n",
       "     {'lower': -31.703169778749995, 'upper': -29.510086223125, 'count': 0},\n",
       "     {'lower': -29.510086223125, 'upper': -27.3170026675, 'count': 0},\n",
       "     {'lower': -27.3170026675, 'upper': -25.123919111874997, 'count': 0},\n",
       "     {'lower': -25.123919111874997, 'upper': -22.930835556250003, 'count': 0},\n",
       "     {'lower': -22.930835556250003, 'upper': -20.737752000625, 'count': 0},\n",
       "     {'lower': -20.737752000625, 'upper': -18.544668445, 'count': 0},\n",
       "     {'lower': -18.544668445, 'upper': -16.351584889374998, 'count': 0},\n",
       "     {'lower': -16.351584889374998, 'upper': -14.158501333749996, 'count': 0},\n",
       "     {'lower': -14.158501333749996, 'upper': -11.965417778125001, 'count': 0},\n",
       "     {'lower': -11.965417778125001, 'upper': -9.7723342225, 'count': 0},\n",
       "     {'lower': -9.7723342225, 'upper': -7.579250666874998, 'count': 0},\n",
       "     {'lower': -7.579250666874998, 'upper': -5.386167111250003, 'count': 0},\n",
       "     {'lower': -5.386167111250003, 'upper': -3.1930835556249946, 'count': 0},\n",
       "     {'lower': -3.1930835556249946, 'upper': -1.0, 'count': 745}],\n",
       "    'count_uniques': 18178},\n",
       "   'dtype': 'object'},\n",
       "  'Location': {'profiler_dtype': 'string',\n",
       "   'stats': {'mismatch': 0,\n",
       "    'missing': 0,\n",
       "    'match': 319073,\n",
       "    'frequency': [{'value': '(0.00000000, 0.00000000)', 'count': 19999},\n",
       "     {'value': '(42.34862382, -71.08277637)', 'count': 1243},\n",
       "     {'value': '(42.36183857, -71.05976489)', 'count': 1208},\n",
       "     {'value': '(42.28482577, -71.09137369)', 'count': 1121},\n",
       "     {'value': '(42.32866284, -71.08563401)', 'count': 1042},\n",
       "     {'value': '(42.25621592, -71.12401947)', 'count': 898},\n",
       "     {'value': '(42.29755533, -71.05970910)', 'count': 783},\n",
       "     {'value': '(42.34128751, -71.05467933)', 'count': 773},\n",
       "     {'value': '(-1.00000000, -1.00000000)', 'count': 745},\n",
       "     {'value': '(42.33152148, -71.07085307)', 'count': 735},\n",
       "     {'value': '(42.35231190, -71.06370510)', 'count': 688},\n",
       "     {'value': '(42.33954199, -71.06940877)', 'count': 655},\n",
       "     {'value': '(42.32696647, -71.06198607)', 'count': 652},\n",
       "     {'value': '(42.35512339, -71.06087980)', 'count': 584},\n",
       "     {'value': '(42.30971857, -71.10429432)', 'count': 573},\n",
       "     {'value': '(42.29848866, -71.06313294)', 'count': 562},\n",
       "     {'value': '(42.33401829, -71.07638124)', 'count': 561},\n",
       "     {'value': '(42.33367922, -71.09187755)', 'count': 550},\n",
       "     {'value': '(42.31043400, -71.06134010)', 'count': 523},\n",
       "     {'value': '(42.35095909, -71.07412780)', 'count': 523},\n",
       "     {'value': '(42.35241815, -71.06525499)', 'count': 515},\n",
       "     {'value': '(42.37081805, -71.03929078)', 'count': 507},\n",
       "     {'value': '(42.33428841, -71.07239518)', 'count': 504},\n",
       "     {'value': '(42.34980175, -71.07840978)', 'count': 472},\n",
       "     {'value': '(42.32696802, -71.08051941)', 'count': 472},\n",
       "     {'value': '(42.33511904, -71.07491710)', 'count': 455},\n",
       "     {'value': '(42.35037870, -71.07626098)', 'count': 445},\n",
       "     {'value': '(42.34653820, -71.09880598)', 'count': 444},\n",
       "     {'value': '(42.36643546, -71.06135413)', 'count': 440},\n",
       "     {'value': '(42.28709355, -71.14822128)', 'count': 438},\n",
       "     {'value': '(42.35602373, -71.06177615)', 'count': 436},\n",
       "     {'value': '(42.34840576, -71.08688339)', 'count': 432},\n",
       "     {'value': '(42.34905600, -71.15049850)', 'count': 427}],\n",
       "    'count_uniques': 18194},\n",
       "   'dtype': 'object'}},\n",
       " 'name': None,\n",
       " 'file_name': 'crime.csv',\n",
       " 'summary': {'cols_count': 17,\n",
       "  'rows_count': 319073,\n",
       "  'dtypes_list': ['object'],\n",
       "  'total_count_dtypes': 1,\n",
       "  'missing_count': 0,\n",
       "  'p_missing': 0.0}}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.ext.profile(\"*\", flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'string'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.meta.get()[\"profile\"][\"columns\"][\"INCIDENT_NUMBER\"][\"profiler_dtype\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INCIDENT_NUMBER</th>\n",
       "      <th>OFFENSE_CODE</th>\n",
       "      <th>OFFENSE_CODE_GROUP</th>\n",
       "      <th>OFFENSE_DESCRIPTION</th>\n",
       "      <th>DISTRICT</th>\n",
       "      <th>REPORTING_AREA</th>\n",
       "      <th>SHOOTING</th>\n",
       "      <th>OCCURRED_ON_DATE</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>MONTH</th>\n",
       "      <th>DAY_OF_WEEK</th>\n",
       "      <th>HOUR</th>\n",
       "      <th>UCR_PART</th>\n",
       "      <th>STREET</th>\n",
       "      <th>Lat</th>\n",
       "      <th>Long</th>\n",
       "      <th>Location</th>\n",
       "      <th>hola</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I182070945</td>\n",
       "      <td>00619</td>\n",
       "      <td>Larceny</td>\n",
       "      <td>LARCENY ALL OTHERS</td>\n",
       "      <td>D14</td>\n",
       "      <td>808</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-02 13:00:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>9</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>13</td>\n",
       "      <td>Part One</td>\n",
       "      <td>LINCOLN ST</td>\n",
       "      <td>42.35779134</td>\n",
       "      <td>-71.13937053</td>\n",
       "      <td>(42.35779134, -71.13937053)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I182070943</td>\n",
       "      <td>01402</td>\n",
       "      <td>Vandalism</td>\n",
       "      <td>VANDALISM</td>\n",
       "      <td>C11</td>\n",
       "      <td>347</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-08-21 00:00:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>8</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>0</td>\n",
       "      <td>Part Two</td>\n",
       "      <td>HECLA ST</td>\n",
       "      <td>42.30682138</td>\n",
       "      <td>-71.06030035</td>\n",
       "      <td>(42.30682138, -71.06030035)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I182070941</td>\n",
       "      <td>03410</td>\n",
       "      <td>Towed</td>\n",
       "      <td>TOWED MOTOR VEHICLE</td>\n",
       "      <td>D4</td>\n",
       "      <td>151</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-03 19:27:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>9</td>\n",
       "      <td>Monday</td>\n",
       "      <td>19</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>CAZENOVE ST</td>\n",
       "      <td>42.34658879</td>\n",
       "      <td>-71.07242943</td>\n",
       "      <td>(42.34658879, -71.07242943)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I182070940</td>\n",
       "      <td>03114</td>\n",
       "      <td>Investigate Property</td>\n",
       "      <td>INVESTIGATE PROPERTY</td>\n",
       "      <td>D4</td>\n",
       "      <td>272</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-03 21:16:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>9</td>\n",
       "      <td>Monday</td>\n",
       "      <td>21</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>NEWCOMB ST</td>\n",
       "      <td>42.33418175</td>\n",
       "      <td>-71.07866441</td>\n",
       "      <td>(42.33418175, -71.07866441)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I182070938</td>\n",
       "      <td>03114</td>\n",
       "      <td>Investigate Property</td>\n",
       "      <td>INVESTIGATE PROPERTY</td>\n",
       "      <td>B3</td>\n",
       "      <td>421</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-03 21:05:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>9</td>\n",
       "      <td>Monday</td>\n",
       "      <td>21</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>DELHI ST</td>\n",
       "      <td>42.27536542</td>\n",
       "      <td>-71.09036101</td>\n",
       "      <td>(42.27536542, -71.09036101)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319068</th>\n",
       "      <td>I050310906-00</td>\n",
       "      <td>03125</td>\n",
       "      <td>Warrant Arrests</td>\n",
       "      <td>WARRANT ARREST</td>\n",
       "      <td>D4</td>\n",
       "      <td>285</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-06-05 17:25:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>17</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>COVENTRY ST</td>\n",
       "      <td>42.33695098</td>\n",
       "      <td>-71.08574813</td>\n",
       "      <td>(42.33695098, -71.08574813)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319069</th>\n",
       "      <td>I030217815-08</td>\n",
       "      <td>00111</td>\n",
       "      <td>Homicide</td>\n",
       "      <td>MURDER, NON-NEGLIGIENT MANSLAUGHTER</td>\n",
       "      <td>E18</td>\n",
       "      <td>520</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-07-09 13:38:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>7</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>13</td>\n",
       "      <td>Part One</td>\n",
       "      <td>RIVER ST</td>\n",
       "      <td>42.25592648</td>\n",
       "      <td>-71.12317207</td>\n",
       "      <td>(42.25592648, -71.12317207)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319070</th>\n",
       "      <td>I030217815-08</td>\n",
       "      <td>03125</td>\n",
       "      <td>Warrant Arrests</td>\n",
       "      <td>WARRANT ARREST</td>\n",
       "      <td>E18</td>\n",
       "      <td>520</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-07-09 13:38:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>7</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>13</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>RIVER ST</td>\n",
       "      <td>42.25592648</td>\n",
       "      <td>-71.12317207</td>\n",
       "      <td>(42.25592648, -71.12317207)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319071</th>\n",
       "      <td>I010370257-00</td>\n",
       "      <td>03125</td>\n",
       "      <td>Warrant Arrests</td>\n",
       "      <td>WARRANT ARREST</td>\n",
       "      <td>E13</td>\n",
       "      <td>569</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-31 19:35:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>19</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>NEW WASHINGTON ST</td>\n",
       "      <td>42.30233307</td>\n",
       "      <td>-71.11156487</td>\n",
       "      <td>(42.30233307, -71.11156487)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319072</th>\n",
       "      <td>142052550</td>\n",
       "      <td>03125</td>\n",
       "      <td>Warrant Arrests</td>\n",
       "      <td>WARRANT ARREST</td>\n",
       "      <td>D4</td>\n",
       "      <td>903</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-06-22 00:12:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>6</td>\n",
       "      <td>Monday</td>\n",
       "      <td>0</td>\n",
       "      <td>Part Three</td>\n",
       "      <td>WASHINGTON ST</td>\n",
       "      <td>42.33383935</td>\n",
       "      <td>-71.08029038</td>\n",
       "      <td>(42.33383935, -71.08029038)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>319073 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       INCIDENT_NUMBER OFFENSE_CODE    OFFENSE_CODE_GROUP  \\\n",
       "0           I182070945        00619               Larceny   \n",
       "1           I182070943        01402             Vandalism   \n",
       "2           I182070941        03410                 Towed   \n",
       "3           I182070940        03114  Investigate Property   \n",
       "4           I182070938        03114  Investigate Property   \n",
       "...                ...          ...                   ...   \n",
       "319068   I050310906-00        03125       Warrant Arrests   \n",
       "319069   I030217815-08        00111              Homicide   \n",
       "319070   I030217815-08        03125       Warrant Arrests   \n",
       "319071   I010370257-00        03125       Warrant Arrests   \n",
       "319072       142052550        03125       Warrant Arrests   \n",
       "\n",
       "                        OFFENSE_DESCRIPTION DISTRICT REPORTING_AREA SHOOTING  \\\n",
       "0                        LARCENY ALL OTHERS      D14            808      NaN   \n",
       "1                                 VANDALISM      C11            347      NaN   \n",
       "2                       TOWED MOTOR VEHICLE       D4            151      NaN   \n",
       "3                      INVESTIGATE PROPERTY       D4            272      NaN   \n",
       "4                      INVESTIGATE PROPERTY       B3            421      NaN   \n",
       "...                                     ...      ...            ...      ...   \n",
       "319068                       WARRANT ARREST       D4            285      NaN   \n",
       "319069  MURDER, NON-NEGLIGIENT MANSLAUGHTER      E18            520      NaN   \n",
       "319070                       WARRANT ARREST      E18            520      NaN   \n",
       "319071                       WARRANT ARREST      E13            569      NaN   \n",
       "319072                       WARRANT ARREST       D4            903      NaN   \n",
       "\n",
       "           OCCURRED_ON_DATE  YEAR MONTH DAY_OF_WEEK HOUR    UCR_PART  \\\n",
       "0       2018-09-02 13:00:00  2018     9      Sunday   13    Part One   \n",
       "1       2018-08-21 00:00:00  2018     8     Tuesday    0    Part Two   \n",
       "2       2018-09-03 19:27:00  2018     9      Monday   19  Part Three   \n",
       "3       2018-09-03 21:16:00  2018     9      Monday   21  Part Three   \n",
       "4       2018-09-03 21:05:00  2018     9      Monday   21  Part Three   \n",
       "...                     ...   ...   ...         ...  ...         ...   \n",
       "319068  2016-06-05 17:25:00  2016     6      Sunday   17  Part Three   \n",
       "319069  2015-07-09 13:38:00  2015     7    Thursday   13    Part One   \n",
       "319070  2015-07-09 13:38:00  2015     7    Thursday   13  Part Three   \n",
       "319071  2016-05-31 19:35:00  2016     5     Tuesday   19  Part Three   \n",
       "319072  2015-06-22 00:12:00  2015     6      Monday    0  Part Three   \n",
       "\n",
       "                   STREET          Lat          Long  \\\n",
       "0              LINCOLN ST  42.35779134  -71.13937053   \n",
       "1                HECLA ST  42.30682138  -71.06030035   \n",
       "2             CAZENOVE ST  42.34658879  -71.07242943   \n",
       "3              NEWCOMB ST  42.33418175  -71.07866441   \n",
       "4                DELHI ST  42.27536542  -71.09036101   \n",
       "...                   ...          ...           ...   \n",
       "319068        COVENTRY ST  42.33695098  -71.08574813   \n",
       "319069           RIVER ST  42.25592648  -71.12317207   \n",
       "319070           RIVER ST  42.25592648  -71.12317207   \n",
       "319071  NEW WASHINGTON ST  42.30233307  -71.11156487   \n",
       "319072      WASHINGTON ST  42.33383935  -71.08029038   \n",
       "\n",
       "                           Location hola  \n",
       "0       (42.35779134, -71.13937053)    5  \n",
       "1       (42.30682138, -71.06030035)    5  \n",
       "2       (42.34658879, -71.07242943)    5  \n",
       "3       (42.33418175, -71.07866441)    5  \n",
       "4       (42.27536542, -71.09036101)    5  \n",
       "...                             ...  ...  \n",
       "319068  (42.33695098, -71.08574813)    5  \n",
       "319069  (42.25592648, -71.12317207)    5  \n",
       "319070  (42.25592648, -71.12317207)    5  \n",
       "319071  (42.30233307, -71.11156487)    5  \n",
       "319072  (42.33383935, -71.08029038)    5  \n",
       "\n",
       "[319073 rows x 18 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df.cols.set(value=\"df['HOUR']+ df['Long']\", output_cols=\"hola\").compute()\n",
    "df.cols.set(value=\"5\", output_cols=\"hola\").compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.ext.repartition(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "string\n",
      "['INCIDENT_NUMBER', 'OFFENSE_CODE']\n",
      "       INCIDENT_NUMBER OFFENSE_CODE    OFFENSE_CODE_GROUP  \\\n",
      "0           I182070945        00619               Larceny   \n",
      "1           I182070943        01402             Vandalism   \n",
      "2           I182070941        03410                 Towed   \n",
      "3           I182070940        03114  Investigate Property   \n",
      "4           I182070938        03114  Investigate Property   \n",
      "...                ...          ...                   ...   \n",
      "319068   I050310906-00        03125       Warrant Arrests   \n",
      "319069   I030217815-08        00111              Homicide   \n",
      "319070   I030217815-08        03125       Warrant Arrests   \n",
      "319071   I010370257-00        03125       Warrant Arrests   \n",
      "319072       142052550        03125       Warrant Arrests   \n",
      "\n",
      "                        OFFENSE_DESCRIPTION DISTRICT REPORTING_AREA SHOOTING  \\\n",
      "0                        LARCENY ALL OTHERS      D14            808      NaN   \n",
      "1                                 VANDALISM      C11            347      NaN   \n",
      "2                       TOWED MOTOR VEHICLE       D4            151      NaN   \n",
      "3                      INVESTIGATE PROPERTY       D4            272      NaN   \n",
      "4                      INVESTIGATE PROPERTY       B3            421      NaN   \n",
      "...                                     ...      ...            ...      ...   \n",
      "319068                       WARRANT ARREST       D4            285      NaN   \n",
      "319069  MURDER, NON-NEGLIGIENT MANSLAUGHTER      E18            520      NaN   \n",
      "319070                       WARRANT ARREST      E18            520      NaN   \n",
      "319071                       WARRANT ARREST      E13            569      NaN   \n",
      "319072                       WARRANT ARREST       D4            903      NaN   \n",
      "\n",
      "           OCCURRED_ON_DATE  YEAR MONTH DAY_OF_WEEK HOUR    UCR_PART  \\\n",
      "0       2018-09-02 13:00:00  2018     9      Sunday   13    Part One   \n",
      "1       2018-08-21 00:00:00  2018     8     Tuesday    0    Part Two   \n",
      "2       2018-09-03 19:27:00  2018     9      Monday   19  Part Three   \n",
      "3       2018-09-03 21:16:00  2018     9      Monday   21  Part Three   \n",
      "4       2018-09-03 21:05:00  2018     9      Monday   21  Part Three   \n",
      "...                     ...   ...   ...         ...  ...         ...   \n",
      "319068  2016-06-05 17:25:00  2016     6      Sunday   17  Part Three   \n",
      "319069  2015-07-09 13:38:00  2015     7    Thursday   13    Part One   \n",
      "319070  2015-07-09 13:38:00  2015     7    Thursday   13  Part Three   \n",
      "319071  2016-05-31 19:35:00  2016     5     Tuesday   19  Part Three   \n",
      "319072  2015-06-22 00:12:00  2015     6      Monday    0  Part Three   \n",
      "\n",
      "                   STREET          Lat          Long  \\\n",
      "0              LINCOLN ST  42.35779134  -71.13937053   \n",
      "1                HECLA ST  42.30682138  -71.06030035   \n",
      "2             CAZENOVE ST  42.34658879  -71.07242943   \n",
      "3              NEWCOMB ST  42.33418175  -71.07866441   \n",
      "4                DELHI ST  42.27536542  -71.09036101   \n",
      "...                   ...          ...           ...   \n",
      "319068        COVENTRY ST  42.33695098  -71.08574813   \n",
      "319069           RIVER ST  42.25592648  -71.12317207   \n",
      "319070           RIVER ST  42.25592648  -71.12317207   \n",
      "319071  NEW WASHINGTON ST  42.30233307  -71.11156487   \n",
      "319072      WASHINGTON ST  42.33383935  -71.08029038   \n",
      "\n",
      "                           Location                hola  \n",
      "0       (42.35779134, -71.13937053)     I18207094500619  \n",
      "1       (42.30682138, -71.06030035)     I18207094301402  \n",
      "2       (42.34658879, -71.07242943)     I18207094103410  \n",
      "3       (42.33418175, -71.07866441)     I18207094003114  \n",
      "4       (42.27536542, -71.09036101)     I18207093803114  \n",
      "...                             ...                 ...  \n",
      "319068  (42.33695098, -71.08574813)  I050310906-0003125  \n",
      "319069  (42.25592648, -71.12317207)  I030217815-0800111  \n",
      "319070  (42.25592648, -71.12317207)  I030217815-0803125  \n",
      "319071  (42.30233307, -71.11156487)  I010370257-0003125  \n",
      "319072  (42.33383935, -71.08029038)      14205255003125  \n",
      "\n",
      "[319073 rows x 18 columns]\n",
      "Wall time: 388 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import pandas as pd\n",
    "from numpy.core._exceptions import UFuncTypeError\n",
    "import numpy as np\n",
    "from numba import njit\n",
    "import fastnumbers\n",
    "import dask.dataframe as dd\n",
    "import re\n",
    "columns = df.cols.names()\n",
    "# col_a = \"id\"\n",
    "# col_b = \"firstName\"\n",
    "\n",
    "cols = None\n",
    "\n",
    "\n",
    "\n",
    "def set(df, op, output_col=None): \n",
    "    \n",
    "    # try to infer if we are going to handle the operations as numeric or string\n",
    "    # Get first column in the operation\n",
    "    columns = [f_col[1:len(f_col)-1] for f_col in re.findall(r\"\\[(['A-Za-z0-9_']+)\\]\", op)]\n",
    "    f_col = columns[0]\n",
    "    \n",
    "    column_dtype = df.cols.profiler_dtypes(f_col)[f_col]\n",
    "    \n",
    "    print(column_dtype)\n",
    "    if column_dtype ==\"num\":\n",
    "        vfunc = lambda x:fastnumbers.fast_float(x, default=np.nan)\n",
    "    elif column_dtype == \"string\" or column_dtype is None:\n",
    "        vfunc = lambda x:str(x)\n",
    "        \n",
    "\n",
    "    def func(pdf, args):  \n",
    "\n",
    "        df = pdf.applymap(vfunc)\n",
    "        try:\n",
    "            return eval(op)\n",
    "        except:\n",
    "            return np.nan\n",
    "    print(columns)\n",
    "    return df.assign(hola=df[columns].map_partitions(func, args= (columns))).compute()\n",
    "\n",
    "print(set(df, \"df['INCIDENT_NUMBER']+df['OFFENSE_CODE']\", \"hola1\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of partitions 8\n",
      "     x   y   z\n",
      "13  13  13  13\n",
      "14  14  14  14\n",
      "15  15  15  15\n",
      "16  16  16  16\n",
      "17  17  17  17\n",
      "18  18  18  18\n",
      "19  19  19  19\n",
      "20  20  20  20\n",
      "21  21  21  21\n",
      "22  22  22  22\n",
      "23  23  23  23\n",
      "24  24  24  24\n",
      "25  25  25  25     x   y   z\n",
      "26  26  26  26\n",
      "27  27  27  27\n",
      "28  28  28  28\n",
      "29  29  29  29\n",
      "30  30  30  30\n",
      "31  31  31  31\n",
      "32  32  32  32\n",
      "33  33  33  33\n",
      "34  34  34  34\n",
      "35  35  35  35\n",
      "36  36  36  36\n",
      "37  37  37  37\n",
      "38  38  38  38\n",
      "99999999999999999999999999999999999999999999999\n",
      "     x   y   z\n",
      "39  39  39  39\n",
      "40  40  40  40\n",
      "41  41  41  41\n",
      "42  42  42  42\n",
      "43  43  43  43\n",
      "44  44  44  44\n",
      "45  45  45  45\n",
      "46  46  46  46\n",
      "47  47  47  47\n",
      "48  48  48  48\n",
      "49  49  49  49\n",
      "50  50  50  50\n",
      "51  51  51  51\n",
      "99999999999999999999999999999999999999999999999\n",
      "     x   y   z\n",
      "0    0   0   0\n",
      "1    1   1   1\n",
      "2    2   2   2\n",
      "3    3   3   3\n",
      "4    4   4   4\n",
      "5    5   5   5\n",
      "6    6   6   6\n",
      "7    7   7   7\n",
      "8    8   8   8\n",
      "9    9   9   9\n",
      "10  10  10  10\n",
      "11  11  11  11\n",
      "12  12  12  12\n",
      "99999999999999999999999999999999999999999999999\n",
      "\n",
      "99999999999999999999999999999999999999999999999     x   y   z\n",
      "52  52  52  52\n",
      "53  53  53  53\n",
      "54  54  54  54\n",
      "55  55  55  55\n",
      "56  56  56  56\n",
      "57  57  57  57\n",
      "58  58  58  58\n",
      "59  59  59  59\n",
      "60  60  60  60\n",
      "61  61  61  61\n",
      "62  62  62  62\n",
      "63  63  63  63\n",
      "64  64  64  64\n",
      "99999999999999999999999999999999999999999999999\n",
      "     x   y   z\n",
      "91  91  91  91\n",
      "92  92  92  92\n",
      "93  93  93  93\n",
      "94  94  94  94\n",
      "95  95  95  95\n",
      "96  96  96  96\n",
      "97  97  97  97\n",
      "98  98  98  98\n",
      "99  99  99  99\n",
      "99999999999999999999999999999999999999999999999\n",
      "\n",
      "     x   y   z\n",
      "65  65  65  65\n",
      "66  66  66  66\n",
      "67  67  67  67\n",
      "68  68  68  68\n",
      "69  69  69  69\n",
      "70  70  70  70\n",
      "71  71  71  71\n",
      "72  72  72  72\n",
      "73  73  73  73\n",
      "74  74  74  74\n",
      "75  75  75  75\n",
      "76  76  76  76\n",
      "77  77  77  77\n",
      "99999999999999999999999999999999999999999999999\n",
      "     x   y   z\n",
      "78  78  78  78\n",
      "79  79  79  79\n",
      "80  80  80  80\n",
      "81  81  81  81\n",
      "82  82  82  82\n",
      "83  83  83  83\n",
      "84  84  84  84\n",
      "85  85  85  85\n",
      "86  86  86  86\n",
      "87  87  87  87\n",
      "88  88  88  88\n",
      "89  89  89  89\n",
      "90  90  90  90\n",
      "99999999999999999999999999999999999999999999999\n",
      "                               0                               1   \\\n",
      "0  {'sum': 0, 'min': 0, 'max': 0}  {'sum': 3, 'min': 1, 'max': 1}   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "\n",
      "                               2                               3   \\\n",
      "0  {'sum': 6, 'min': 2, 'max': 2}  {'sum': 9, 'min': 3, 'max': 3}   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "0                             NaN                             NaN   \n",
      "\n",
      "                                4                                5   \\\n",
      "0  {'sum': 12, 'min': 4, 'max': 4}  {'sum': 15, 'min': 5, 'max': 5}   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "\n",
      "                                6                                7   \\\n",
      "0  {'sum': 18, 'min': 6, 'max': 6}  {'sum': 21, 'min': 7, 'max': 7}   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "0                              NaN                              NaN   \n",
      "\n",
      "                                8                                9   ...  \\\n",
      "0  {'sum': 24, 'min': 8, 'max': 8}  {'sum': 27, 'min': 9, 'max': 9}  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "0                              NaN                              NaN  ...   \n",
      "\n",
      "                                   90                                  91  \\\n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0  {'sum': 270, 'min': 90, 'max': 90}                                 NaN   \n",
      "0                                 NaN  {'sum': 273, 'min': 91, 'max': 91}   \n",
      "\n",
      "                                   92                                  93  \\\n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0  {'sum': 276, 'min': 92, 'max': 92}  {'sum': 279, 'min': 93, 'max': 93}   \n",
      "\n",
      "                                   94                                  95  \\\n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0  {'sum': 282, 'min': 94, 'max': 94}  {'sum': 285, 'min': 95, 'max': 95}   \n",
      "\n",
      "                                   96                                  97  \\\n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0                                 NaN                                 NaN   \n",
      "0  {'sum': 288, 'min': 96, 'max': 96}  {'sum': 291, 'min': 97, 'max': 97}   \n",
      "\n",
      "                                   98                                  99  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0                                 NaN                                 NaN  \n",
      "0  {'sum': 294, 'min': 98, 'max': 98}  {'sum': 297, 'min': 99, 'max': 99}  \n",
      "\n",
      "[8 rows x 100 columns]\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "import functools\n",
    "import dask\n",
    "import dask.dataframe as dd\n",
    "import pandas as pd\n",
    "\n",
    "pdf = pd.DataFrame({\n",
    "    'x': range(0, 100),\n",
    "    'y': range(0, 100),\n",
    "    'z': range(0, 100)\n",
    "})\n",
    "\n",
    "ddf = dd.from_pandas(pdf, npartitions=8)\n",
    "\n",
    "print('Number of partitions', ddf.npartitions)\n",
    "\n",
    "\n",
    "def compute_stats(row):\n",
    "    return {\n",
    "        'sum': row['x'] + row['y'] + row['z'],\n",
    "        'min': min(row),\n",
    "        'max': max(row)\n",
    "    }\n",
    "\n",
    "\n",
    "def accum_stats(stats_accum, stats):\n",
    "    return {\n",
    "        'sum': stats_accum['sum'] + stats['sum'],\n",
    "        'min': min(stats_accum['min'], stats['min']),\n",
    "        'max': max(stats_accum['max'], stats['max'])\n",
    "    }\n",
    "\n",
    "\n",
    "def compute_stats_partition(pdf):\n",
    "    pds = pdf.apply(compute_stats, axis=1)\n",
    "    print(pdf)\n",
    "    print(\"99999999999999999999999999999999999999999999999\")\n",
    "    return pds\n",
    "    return functools.reduce(accum_stats, pds)\n",
    "\n",
    "\n",
    "def merge_stats_series(pds):\n",
    "    print(pds)\n",
    "    print(\"--\")\n",
    "    return pds\n",
    "    return functools.reduce(accum_stats, pds)\n",
    "\n",
    "\n",
    "res = ddf.reduction(\n",
    "    compute_stats_partition,\n",
    "    merge_stats_series,\n",
    "    meta={\n",
    "        'sum': 'int64',\n",
    "        'min': 'int64',\n",
    "        'max': 'int64'\n",
    "    })\n",
    "\n",
    "# singleton dataframe to list of delayed objects\n",
    "# where each row is a delayed object\n",
    "# and in this case we just want the first one\n",
    "delayed_dict = res.to_delayed()[0]\n",
    "a = dd.compute(delayed_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
